/**
 * Copyright (c) 2008, Piercarlo Slavazza
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * The name of Piercarlo Slavazza cannot be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY Piercarlo Slavazza ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL Piercarlo Slavazza BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package com.libapml.indexer;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.InvalidParameterException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.store.FSDirectory;
import org.ontoware.aifbcommons.collection.ClosableIterator;
import org.ontoware.rdf2go.RDF2Go;
import org.ontoware.rdf2go.exception.ModelRuntimeException;
import org.ontoware.rdf2go.model.Model;
import org.ontoware.rdf2go.model.Statement;
import org.ontoware.rdf2go.model.node.impl.URIImpl;
import org.semanticdesktop.aperture.extractor.Extractor;
import org.semanticdesktop.aperture.extractor.ExtractorException;
import org.semanticdesktop.aperture.extractor.ExtractorFactory;
import org.semanticdesktop.aperture.extractor.ExtractorRegistry;
import org.semanticdesktop.aperture.extractor.impl.DefaultExtractorRegistry;
import org.semanticdesktop.aperture.rdf.RDFContainer;
import org.semanticdesktop.aperture.rdf.impl.RDFContainerImpl;
import org.semanticdesktop.aperture.vocabulary.NIE;

import com.libapml.ApmlException;
import com.libapml.util.Helper;
import com.libapml.util.IOUtil;
import com.libapml.util.ReaderInputStream;
import com.sun.syndication.feed.synd.SyndEntry;
import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.io.FeedException;
import com.sun.syndication.io.ParsingFeedException;
import com.sun.syndication.io.SyndFeedInput;
import com.sun.syndication.io.XmlReader;

public class ApmlIndexer {

	public static final String INDEX_PATH = "./lucene_index/";
	
	private LuceneWriter luceneWriter;

	public ApmlIndexer() throws IOException {
		this(false);
	}
	
	public ApmlIndexer(boolean useStopwords) throws IOException {
		
		IOUtil.deleteDirectory(new File(INDEX_PATH));
		
		luceneWriter = new LuceneWriter(
				FSDirectory.getDirectory(INDEX_PATH),
				(useStopwords ? loadStopWords() : null));
		
	}
	
	@SuppressWarnings("unchecked")
	private Set loadStopWords() throws IOException {
		
		InputStream is = 
			LuceneWriter.class.getClassLoader().getResource("stop_words_en.txt").openStream();
		try {
			return WordlistLoader.getWordSet(new InputStreamReader(is));
		} finally {
			if (is != null)
				is.close();
		}
		
	}
	
	public LuceneWriter getLuceneWriter() {
		return luceneWriter;
	}
	
	@SuppressWarnings("unchecked")
	private String getContent(String content, String uriValue, String mimeType) throws ExtractorException, ModelRuntimeException, IOException {
		
		String parsedContent = "";
		
        ExtractorRegistry extractorRegistry = new DefaultExtractorRegistry();
		
		org.ontoware.rdf2go.model.node.URI uri = new URIImpl(uriValue);
        Model model = RDF2Go.getModelFactory().createModel();
        model.open();
        RDFContainer container = new RDFContainerImpl(model, uri);
        // determine and apply an Extractor that can handle this MIME type
        Set factories = extractorRegistry.get(mimeType);
        if (factories != null && !factories.isEmpty()) {
            ExtractorFactory factory = (ExtractorFactory) factories.iterator().next();
            Extractor extractor = factory.get();
 
            extractor.extract(uri, 
            		new ReaderInputStream(new StringReader(content)), 
            		null, mimeType, container);
        }
        container.add(NIE.mimeType, mimeType);
        //container.getModel().writeTo(new PrintWriter(System.out),Syntax.Ntriples);

        ClosableIterator<Statement> statements = container.getModel().iterator();
        try {
        	while (statements.hasNext()) {
        		Statement statement = (Statement) statements.next();

        		if ("http://www.semanticdesktop.org/ontologies/2007/01/19/nie#plainTextContent".equalsIgnoreCase(statement.getPredicate().toString())) {
        			parsedContent = statement.getObject().asLiteral().getValue();
        			break;
        		}
        	}
        } finally {
        	statements.close();
        }
		container.getModel().close();
        
        return parsedContent;
	}
	
	@SuppressWarnings("unchecked")
	private List<ApmlDocument> getDocuments(File feedFile) throws IllegalArgumentException, FeedException, IOException, URISyntaxException, ModelRuntimeException, ExtractorException {
		
		List<ApmlDocument> feedDocuments = new ArrayList<ApmlDocument>();
		
        try {
        	
        	SyndFeedInput input = new SyndFeedInput();
        	SyndFeed feed = input.build(new XmlReader(feedFile));
        	
        	List<SyndEntry> entries = feed.getEntries();
        	int i = 0;
        	for (SyndEntry entry : entries) {
        		ApmlDocument doc = new ApmlDocument();
        		doc.uri = new URI(feedFile.toURI() + "#" + i++);
				
        		String description = "";
        		if (entry.getDescription() != null)
        			description = entry.getDescription().getValue();
				doc.content = getContent(
						entry.getTitle() + " " + description, 
						doc.uri.toString(), "text/html");
				
				feedDocuments.add(doc);
			}
        	
        	System.out.println("FEED PARSED!");
        	
        } catch (ParsingFeedException e) {
			System.out.println("==> NOT A FEED!");
		} catch (FileNotFoundException e) {
			System.out.println(e.getMessage());;
		}
		
        return feedDocuments;
		
	}
	
	private void addFile(File file) throws ApmlException {
		
		System.out.println("Indexing file: " + file);

		try {
			List<ApmlDocument> documents = getDocuments(file); 
			for (ApmlDocument apmlDocument : documents)
				try {
					System.out.println("Adding doc: " + apmlDocument.uri);
					luceneWriter.add(apmlDocument);
				} catch (Exception e) {
					e.printStackTrace();
				}
		} catch (Exception e) {
			throw new ApmlException(e);
		}
			
	}
	
	private void addFolder(File folder) throws ApmlException {
		
		File[] files = folder.listFiles();
		for (int i = 0; i < folder.listFiles().length; i++)
			addFile(files[i]);
		
	}
	
	public void add(File fileOrDirectory) throws ApmlException {
		
		Helper.checkNotNull(fileOrDirectory);
		
		if (!fileOrDirectory.exists()) {
			throw new InvalidParameterException("Folder " + fileOrDirectory.getPath() + " does not exist.");
		} else {
			if (fileOrDirectory.isDirectory())
				addFolder(fileOrDirectory);
			else
				addFile(fileOrDirectory);
		}
		
	}
	
	public void close() throws ApmlException {
		
		try {
			luceneWriter.close();
		} catch (Exception e) {
			throw new ApmlException(e);
		}
		
	}
	
}
